#!/opt/python-2.5/bin/python2.5

## Revelator Module Tutor
## Version 1.0.5
## Written by Josh "Lain" Johnson, 2 August 2009
## Modification 1.0.1:  14 September 2009
## Modification 1.0.2 for first demo:  30 September 2009
## Modification 1.0.3 for CGI:  16 October 2009
## Modification 1.0.4 to merge all code into one program:  19 October 2009
## Modification 1.0.5 to include auxiliary pages and have more Web-friendly formatting:  11 December 2009
## All lefts reserved.

## Invocation (Usage):  ~ tagfile.crf grammar.cfg propts_file.prmz definitions.best init_Topic Web_Flag Trace

import cPickle, nltk, os, re, shelve, string, sys, tempfile
from nltk import grammar, tag, corpus, data
from nltk.parse import EarleyChartParser
from nltk.tag import crf
from nltk.corpus import wordnet

## Self-developed modules:
# import hmmtagger
# import preterms
## These following have been commented out since I have merged them into the Tutor.
# import ask
# from ask import getPlay
# import answer
# from answer import write
# import bestdefs
# import seqtagger
# import tokenize
# import prompt
# import define
# import extract
# import commands
# import getnps

## Path modification, so that the Tutor can locate WordNet resources.
nltk.data.path.append('/corpora/nltk/nltk-data/')

class Tutor:
	"""This will parse a sentence according to a grammar supplied via a gold standard.  If there is a structural 
		ambiguity, which occurs when a sentence has more than one distinct parse, then a customized error message
		is output to the user.  Otherwise, the sentence is converted into a Controlled Natural Language (CNL) format."""

	def __init__(self, Tagfile = "dummy.crf", Grammar = "dummy.cfg", Promptings = "dummy.prmz", Defsfile = "dummy.best",\
					Topic = "Potpourri", venue = 2, TRON = 0):
		"""Initialize the grammar by first loading in the file of acceptible Evidence Implies Inference (EII) format parses,
			then reading in and reformatting the input sentence, before testing its acceptibility and printing output
			accordingly."""

		if (TRON > 2):	
			print Tagfile, Grammar, Promptings, Defsfile, Topic, venue, TRON

		## Zerost, set the file and parameter defaults.
		self.font = "Verdana"
		self.color = "99CCFF"
		self.bgcolor = "333366"
		self.border = "666699"
		self.titleFont = "Trajan Pro"
		self.titleColor = "DDFFDD"
		self.saluFont = "Georgia"
		self.saluColor = "FF99CC"
		self.errorColor = "FF6666"
		self.goodColor = "66FF66"
		self.linkColor = "9999FF"
		self.visitedColor = "CC66CC"

		## These are constant, so they need not be written to any datafile.
		self.nolines = 0
		self.N = 4
		self.for_IM = (venue == 1)
		self.noun_tags = ["NN", "NNS", "FW"]
		self.for_Web = (venue == 2)
		if (self.for_Web):
			import cgi, cgitb
			cgitb.enable()
			print "Content-type: text/html;charset=utf-8"
			print 
			print "<script type=\"text/javascript\">"
			print "function help() { window.open(\"help.html\") }"
			print "function ver() { window.open(\"version.html\") }"
			print "function about() { window.open(\"about.html\") }"
			print "function sample() { window.open(\"sample.html\") }"
			print "</script>"
			print "<body bgcolor=%s text=%s link=%s vlink=%s>" % (self.bgcolor, self.color, self.linkColor, self.visitedColor)
			self.form = cgi.FieldStorage()
			self.player = "demo"
			if "Player" in self.form.keys():
				self.player = self.form["Player"].value
			print "<form name=\"Tutor\" method=post action=\"Tutor.cgi\">"
			print "<input type=hidden name=\"Player\" value=\"" + self.player + "\"/>"

		## These are initialized to Nothing, since they will be either overwritten or not needed.  It prevents
		##   calling errors if they are least treated this way.
		altered = dict()
		words = dict()
		EII_grammar = None
		self.EII_Earley = None
		str = ''
		if_clause = ''
		then_clause = ''
		pos_if = ''
		pos_then = ''
		allNPs = dict()
		self.dummyFile = "tempdir/" + self.player + ".dat"
		self.debug_file = "tempdir/" + self.player + ".skuld"
		self.output_file = "tempdir/" + self.player + ".out"

		## We will then assign everything else, reading the data from a file if this is not our initial run.
		try:
			vars = shelve.open(self.dummyFile, "r")
			self.flag = vars["flag"]
			self.tagfile = vars["tagfile"]
			self.grammar = vars["grammar"]
			self.promptings = vars["promptings"]
			self.defsfile = vars["defsfile"]
			self.old_topic = vars["old_topic"]
			self.new_topic = self.old_topic
			self.promptText = vars["promptText"]
			self.ideal = vars["ideal"]
			self.LISTEN = vars["LISTEN"]
			self.DEFINE = vars["DEFINE"]
			self.CLARIFY = vars["CLARIFY"]
			self.DONE = vars["DONE"]
			altered = vars["altered"]
			words = vars["words"]
			(self.SeqTagger, self.train_data) = (vars["seqtagger"], vars["train_data"])
			(self.messages, self.transforms) = (vars["messages"], vars["transforms"])
			received = vars["received"]
			EII_grammar = vars["eii_grammar"]
			self.EII_Earley = vars["eii_earley"]
			str = vars["str"]
			if_clause = vars["if_clause"]
			then_clause = vars["then_clause"]
			pos_if = vars["pos_if"]
			pos_then = vars["pos_then"]
			allNPs = vars["allnps"]
			valid_if = vars["if"]
			valid_then = vars["then"]
			cI = vars["choiceI"]
			cE = vars["choiceE"]
			self.headinginference = vars["headinginference"] 
			self.headingevidence = vars["headingevidence"]
			self.choicesinference = vars["choicesinference"]
			self.choicesevidence = vars["choicesevidence"]
			playnum = vars["playnum"]
			self.player = vars["player"]
			initial = False
			vars.close()
			# print "<i>Non-initial run</i>"
		except:
			# print "<i>Initial run</i>"
			initial = True
			self.tagfile = Tagfile
			self.grammar = Grammar
			self.promptings = Promptings
			self.defsfile = Defsfile
			self.old_topic = Topic
			self.new_topic = self.old_topic
			self.noun_tags = ["NN", "NNS", "FW"]
			self.promptText = "Please make your play here, in the form of an if-then conjecture.<br/>"
			valid_if = None
			valid_then = None

			## These are for the radio button used for ambiguity resolution.
			cI = None
			cE = None
			self.headinginference = ""
			#	"Choices for resolving ambiguities in your inference.  Ignore me if there's nothing below."
			self.headingevidence = ""
			#	"Choices for resolving ambiguities in your evidence.  Ignore me if there's nothing below."
			self.choicesinference = None
			self.choicesevidence = None
	
			## These flags indicate what event should be triggered, since all of the following is synchronous for Webpages.
			## If self.for_Web is False, this will do nothing harmful whatsoever.
			self.LISTEN = True
			self.DEFINE = True
			self.CLARIFY = True
			self.DONE = False

			## Flag for whether this is the initial run, used for resolving word definitions.
			initial = True

			altered = dict()
			words = dict()

			## First, read in the grammar file.
			str = ''
			gramfile = open(self.grammar, 'rb')
			for line in gramfile.readlines():
				str += line
			# str = cPickle.load(gramfile)
			gramfile.close()

			## Then, initialize the tagger.
			# HMMT = hmmtagger.getHMMTagger(self.tagfile, self.nolines)
			# self.HMMTagger = HMMT.trainHMM()
			(self.SeqTagger, self.train_data) = self.getTagger(self.tagfile, self.nolines, self.N)
			if (TRON > 1):	self.write("<i><tt>This tagger is %0.4f%% accurate.</tt></i>" % self.score(), self.debug_file)

			## Next, we will read in the ambiguous sentence prompts and conversions.
			(self.messages, self.transforms) = self.init_messages(self.promptings)
			self.ideal = self.getBest(self.defsfile)
			
			## Keep track of how many if-then plays the player has made.
			playnum = 1

			received = None 

## --------- END OF INITIALIZATION DATA -----------	

		## Salutation
		## Possibility 2:  An if-then statement has been made, as well as clarification of ambiguities.
		if ("choiceevidence" in self.form.keys()):
			cE = int(self.form["choiceevidence"].value) - 1
			# self.write("Evidence choice: " + repr(cE), self.debug_file)
			valid_if = self.EIItoCNL(if_clause, pos_if, cE)
			self.CLARIFY = False
		if ("choiceinference" in self.form.keys()):
			cI = int(self.form["choiceinference"].value) - 1
			# self.write("Inference choice: " + repr(cI), self.debug_file)
			valid_then = self.EIItoCNL(then_clause, pos_then, cI)
			self.CLARIFY = False
		if 1:
			## Titular display:
			if self.for_Web:	
				print "<h1 align=center><font face=\"%s\" size=7 color=%s>" % (self.titleFont, self.titleColor)
				print "Welcome to Revelator, %s!" % self.player
				print "</font></h1>"
			else:
				self.write("Welcome to Revelator, %s!" % self.player, "screen")
			if ("Quit" in self.form.keys()):
				self.exit()
			received = self.getPlay(self.promptText)
		
			## After that, read in the play or commands from the user.  For now, we will take only one at a time.  
			## We will keep taking input until the user wishes to quit.

			## Proceed iff we have data from the user.

			## Possibility 0:  Nothing has been input.
			# if (not self.DONE and self.CLARIFY and not received):
			#	self.write(self.promptText)
			if received:
				## Possibility 1:  A command, not an if-then statement, was input.
				# ternary = self.aCommand(received)
				# if (ternary == 0):
				self.write(self.player + " " + repr(playnum) + ":  " + received, self.debug_file)
				playnum += 1
				# if (ternary == -1):
					## Salutation
				#	self.exit()
				# if (ternary == 1):
				# 	self.LISTEN = True
				## Possibility 3;  An if-then statement has been made, but we have not parsed it as yet.
				if self.CLARIFY and not self.LISTEN:
					## We can self-define words in this part of the code.
					## And this lets us self-define words.
					(received, words, altered) = self.paran(received, words, altered, self.ideal)
		
					## N. B. that we will tokenize and tag the input twice, once before and once after, so that 
					##   we can extract the paranthetical definitiions and not have them affect the final tagging.
					(words, altered, allNPs, received) = self.call_paranNP(received, words, altered, self.ideal, TRON > 2)
	
					## Add the terminals to the grammar.  We will parse the if- and then-clauses separately.
					(if_clause, then_clause, str, pos_if, pos_then) = self.listen(str, received)
		
					## Now, we get to the grammar.
					EII_grammar = grammar.parse_cfg(str)
					self.EII_Earley = EarleyChartParser(EII_grammar, trace = TRON)
		
					## Following that, we (re)define common nouns.
					if self.DEFINE:
						# (altered, self.ideal, words) = self.get_defs(if_clause.split(), pos_if.split(), altered, self.ideal, words, initial, allNPs)
						# (altered, self.ideal, words) = self.get_defs(then_clause.split(), pos_then.split(), altered, self.ideal, words, initial, allNPs)
						self.DEFINE = False
		
					## Verify the defintions (debug only).
					for NP in allNPs.values():
						if (NP in words):	self.write("NP " + NP + " is defined as " + words[NP] + ".", self.debug_file)
						else:	self.write("NP " + NP + " is heretofore undefined.", self.debug_file)

					## Then, test this sentence against our grammar.
					valid_if = self.parse(if_clause, pos_if, "evidence", cE)
					valid_then = self.parse(then_clause, pos_then, "inference", cI)
					if (valid_if and valid_then):	self.CLARIFY = False

				## If valid, print a copy of the evidence and inference to STDOUT.
				##    Otherwise, tell the user what was wrong.
				# if valid_if and not valid_then:
				#	self.write("I was able to understand your evidential statement, but your inferential statement did not parse.  Could you please restate\nyour entire play?")
				# elif valid_then:
				#	self.write("I was able to understand your statement of inference, but your statement of evidence did not parse.  Could you please restate\nyour entire play?")
				# else:
				#	self.write("I was unable to understand either your inference or evidence.  Please remake your play for me.")

			## Possibility 4:  We've resolved the ambiguous if-then statement.
			if not self.CLARIFY:
				## Reset defaults.
				if (valid_if and valid_then):
					self.printEI(valid_if, valid_then, altered, words, allNPs)
				else:
					self.write("<br/><i>Please retype your play.  I couldn't understand what you meant.</i>", textColor = self.errorColor)
				self.DEFINE = True
				self.CLARIFY = True
				self.LISTEN = True
				self.DONE = False
				received = None
				cE = None
				self.headingevidence = ""
				#	"Choices for resolving ambiguities in your evidence.  Ignore me if there's nothing below."
				self.choicesevidence = None
				self.headinginference = ""
				#	"Choices for resolving ambiguities in your inference.  Ignore me if there's nothing below."
				cI = None
				self.choicesinference = None
				self.promptText = "Please make your play here, in the form of an if-then conjecture.<br/>"
			## Possibility 5:  We have nothing.
			if (not self.DONE and self.CLARIFY):
				## Form for accepting data.  For this incarnation, we're using a text box.
				print "<button name=\"dummyR\" type=reset>Clear play</button>\t"
				if not self.LISTEN:	print "\t<text name=\"ghost\" cols=50 rows=1 readonly>", received, "</text>\t\t"
				else:	print "<input name=\"play\" type=text size=100 maxlength=70 autocomplete/>\t"
				print "<button name=\"dummyS\" type=submit/>Make play</button><br/>"
				print "<br/>" * 1
				if self.headingevidence:	self.write(self.headingevidence, "screen")
				if self.choicesevidence:
					choices = self.choicesevidence.split("\n")[:-1]			
					for x in range(1, len(choices) + 1):
						##         ^-- This is so that a return value of 0 isn't treated as a Fallacy
						print choices[x - 1] + "<input type=radio name=\"choiceevidence\" value=" + repr(x) + " /><br/>"
					print "None of the above."
					print "<input type=radio name=\"choiceevidence\" value=-2 /><br/>"
				if self.headinginference:	self.write(self.headinginference, "screen")
				if self.choicesinference:
					choices = self.choicesinference.split("\n")[:-1]			
					for x in range(1, len(choices) + 1):
						print choices[x - 1] + "<input type=radio name=\"choiceinference\" value=" + repr(x) + " /><br/>"
					print "None of the above."
					print "<input type=radio name=\"choiceinference\" value=-2 /><br/>"
				if (not self.LISTEN):
					print "<button type=submit name=\"dummyRS\">Resubmit play</button><br/>"
				print "<input type=button name=\"Help\" onClick=\"help()\" value=\"Help me out!\" />"
				print "<input type=button name=\"Ver\" onClick=\"ver()\" value=\"Version info\" />"
				print "<input type=button name=\"About\" onClick=\"about()\" value=\"About Revelator\" />"
				print "<input type=button name=\"Sample\" onClick=\"sample()\" value=\"Show me a sample conjecture.\" /><br/><br/>"
				print "<input type=submit name=\"Quit\" value=\"I want to quit playing Revelator.\" />"
				print "</form>"
				self.LISTEN = False
		print "<br/><br/><hr size=3 align=center color=%s /><br/>" % self.border
		## Print out the messages from the Tutor.
		try:
			out = open(self.output_file, 'r')
			self.write("<u><i>Here is the previous play you made.  If I had any concerns with it, you can read them here.</i></u><br/>", "screen", size = 4)
			for line in out:	print line
			out.close()
			try:
				os.remove(self.output_file)
			except:	pass
		except:	pass
		print "</body>"
		## Load up the Shelf.
		self.writeDummy(altered, words, received, EII_grammar, str, if_clause, then_clause, pos_if, pos_then, allNPs, valid_if, valid_then, cE, cI, playnum)


	def listen(self, grammar_string, play):
		"""Runs the input play from the user and runs it through a POS tagger."""
		## This gets rid of unnecessary punctuation then tokenize input from the user.
		# sent = tokenize.tokenize(self.strip_punc(getPlay(" USER>\t")), 0)
		sent = self.tokenize(play, 0)
		self.write("", "screen")
		
		## Default POS tagger:  Stanford POS tagger
		## TODO 2:  Use the Stanford tagger.
		## Backup tagger:  Mallet CRF tagger.
		# MT = crf.MalletCRF('/home/lain0523/tmp/Rev/test_model_1')
		# tagging = MT.tag(sent.split())
		
		## Second backup tagger:  HMM tagger.
		# tagging = self.HMMTagger.tag(sent.split())
       
		## Third backup tagger:  Sequential tagger.
		##   (ASIDE:  The number of taggers is getting ridiculous.)

		## We are parsing the if-clause and then-clause separately.

		(ifc, thenc) = self.getEI(sent)
		# print ifc, thenc
		tagging_if = self.SeqTagger.tag(ifc.split())
		tagging_then = self.SeqTagger.tag(thenc.split())
		tags_if = ""
		tags_then = ""

		## Add the terminals to our grammar string:
		for (token, tag) in tagging_if:
			term = tag + ' -> "' + token + '"\n'
			tags_if += tag + " "
			# print "Tagged", term[:-1]
			grammar_string += term
			# print "Adding", tag, '-> "' + token + '" to the grammar.'
			# self.write("Adding " + tag + ' -> "' + token + '" to the grammar.', self.debug_file)
		for (token, tag) in tagging_then:
			term = tag + ' -> "' + token + '"\n'
			tags_then += tag + " "
			# print "Tagged", term[:-1]
			grammar_string += term
			# self.write("Adding " + tag + ' -> "' + token + '" to the grammar.', self.debug_file)
		return (ifc, thenc, grammar_string, tags_if[:-1], tags_then[:-1])

	def no_parses(self, EorI, entry = "TEXT OMITTED"):
		"""Prints an error message if the translator cannot make any sense of the sentence."""
		# self.write("TUTOR>  I was unable to parse your " + EorI + ".  Please rephrase it for me, so that it is easier to understand.\n", self.debug_file)
		# self.write("<i>I was unable to parse your " + EorI + ".  Please rephrase it for me, so that it is easier to understand.<br/></i>", textColor = self.errorColor)
		self.write("Most of your play looked fine to me.<br/>However, I was unable to understand what you meant by '%s'.  Could you please rephrase your entire play for me?" % entry, textColor = self.errorColor, size = 2)
		self.write("Most of your play looked fine to me.<br/>However, I was unable to understand what you meant by '%s'.  Could you please rephrase your entire play for me?" % entry, self.debug_file)

	def mult_parses(self, EorI, entry = "TEXT OMITTED"):
		"""Prints an error message in case we have structural ambiguity."""
		# self.write("TUTOR>  There were many ways of understanding the " + EorI + " you typed.\n")
		# self.write("TUTOR>  There were many ways of understanding the " + EorI + " you typed.\n", self.debug_file)
		self.write("There are many ways of interpreting the " + EorI + " you typed, '%s'." % entry, size = 2)
		self.write("There are many ways of interpreting the " + EorI + " you typed, '%s'." % entry, self.debug_file)

	def EIItoCNL(self, clause, tags, index):
		"""Converts the EII-formed string into the CNL format."""
		if (index == -3):	return False
		if (index == -4):	return clause
		return self.fitInto(clause, self.transforms[tags][index])
	
	def score(self):
		"""Give an accuracy score (on a [0, 100] scale)."""
		return 100.0 * self.SeqTagger.evaluate(self.train_data)

	def parse(self, clause, tag_seq, EorI, choiceE = None, choiceI = None):
		"""Verify that the clauses parse."""
		parses = self.EII_Earley.nbest_parse(clause.split())
		# print '"' + tag_seq + '"'
		poss = len(parses)
		fitting = -4
		if (poss == 0):
			## We can't parse this clause using our grammar.
			self.no_parses(EorI, clause)
			self.LISTEN = True
			self.CLARIFY = True
			self.DONE = False
			return False
		if (poss > 1):
			## We have multiple parses.  First, we try and use our templates to resolve ambiguities.
			self.mult_parses(EorI, clause)
			# print parses[0], "\n"
			if (tag_seq not in self.messages.keys()):
				## We don't have a template, but we will let this clause pass anyway.
				self.write("Since I can't see any ambiguous meaning, I'll leave it as is, then.", self.debug_file)
				self.write("Since I can't see any ambiguous meaning, I'll leave it as is, then.", size = 2)
				return self.EIItoCNL(clause, tag_seq, -4)
			else:	
				## We here prompt the user to confirm one of various readings of the clause.
				if (self.for_Web):
					# if (choiceE and EorI == "evidence"):
					# 	fitting = choiceE
					# elif (choiceI and EorI == "inference"):
					#	fitting = choiceI
					# else:
					send = ""
					for fitting in range(len(self.messages[tag_seq])):
						send += self.fitInto(clause, self.messages[tag_seq][fitting]) + "\n"
					exec("self.choices" + EorI + " = send")
					exec("self.heading" + EorI + " = \"Please clarify your %s for me.\"" % EorI)
					return False
				else:
					chosen = False
					for fitting in range(len(self.messages[tag_seq])):
						template = self.messages[tag_seq][fitting]
						yn = self.getPlay(self.fitInto(clause, template) + "  ")
						if (self.yes(yn)):
							# print "Going with template", template
							chosen = True
							break			
					if (not chosen):
						## The user has not chosen one, which means we're confused.  Let's have him rephrase it, then.
						self.write("Could you please rephrase your " + EorI + ", since I can't understand your intended meaning?", face = self.font, textColor = self.errorColor)
						return False
					## This is acceptible, so we print the output as a CNL string.
					try:	self.write(choiceE + " " + choiceI)
					except:	self.write("No choices have been recorded or made.")
					asCNL = self.EIItoCNL(clause, tag_seq, fitting)
					self.write("Output of " + EorI + " to Revelator:  " + asCNL)
					return asCNL 

	def get_defs(self, tokens, pos, alt, best, lex, first, NPdict):
		"""(Re)Define the words in the if/then clause provided."""
		if (len(tokens) != len(pos)):	
			self.write("ERROR:  The tokenized sentence doesn't have fully-matching POS tags.", self.debug_file)
			return (alt, best, lex)
		if (first):
			for np in NPdict.values():
				if (np not in lex):
					alt[np] = True
					if (np not in best):	best[np] = 1
					try:
						## This last loop assumes a plural noun ending in 'ies', looking up the base word accordingly.
						try:
							## This one assumes it's a plural noun ending in 'es', and looks up the base word.
							try:
								## This one assumes that, if it ends in 's', it's a plural noun and looks up the base word.
								try:
									## This loop tries just the word itself (executed first).
									lex[np] = wordnet.synset(np + ".n." + str(best[np])).definition
								except (nltk.corpus.reader.wordnet.WordNetError):
									if (np[-1] == "s"):	lex[np] = wordnet.synset(np[:-1] + ".n." + str(best[np])).definition
									else:	raise(nltk.corpus.reader.wordnet.WordNetError)
							except (nltk.corpus.reader.wordnet.WordNetError):
								if (np[-2:] == "es"):	lex[np] = wordnet.synset(np[:-2] + ".n." + str(best[np])).definition
								else:	raise(nltk.corpus.reader.wordnet.WordNetError)
						except (nltk.corpus.reader.wordnet.WordNetError):
							if (np[-3:] == "ies"):	lex[np] = wordnet.synset(np[:-3] + "y.n." + str(best[np])).definition
							else:	raise(nltk.corpus.reader.wordnet.WordNetError)
					except (nltk.corpus.reader.wordnet.WordNetError):
						if not self.for_Web:
							(alt[np], lex[np]) = self.define(np, "", best[np], False, self.old_topic, self.new_topic)
						else:	pass
			# for x in range(len(tokens)):
			#	word = tokens[x]
			#	tag = pos[x]
			#	if ((tag in self.noun_tags) and (word not in lex)):
			#		alt[word] = True
			#		if (word not in best):	best[word] = 1
			#		lookup = word + ".n." + str(best[word])
			#		try:
			#			lex[word] = wordnet.synset(lookup).definition
			#		except(Exception):
			#			(alt[word], lex[word]) = define.define(word, "", best[word], False, self.old_topic, self.new_topic)
			return (alt, best, lex)
		if (self.old_topic == self.new_topic):
			alt = dict()
		for np in NPdict.values():
			if (np not in lex):
				alt[np] = True
				if (np not in best):	best[np] = 1
				try:
					try:
						try:
							try:
								lex[np] = wordnet.synset(np + ".n." + str(best[np])).definition
							except (nltk.corpus.reader.wordnet.WordNetError):
								if (np[-1] == "s"):	lex[np] = wordnet.synset(np[:-1] + ".n." + str(best[np])).definition
								else:	raise(nltk.corpus.reader.wordnet.WordNetError)
						except (nltk.corpus.reader.wordnet.WordNetError):
							if (np[-2:] == "es"):	lex[np] = wordnet.synset(np[:-2] + ".n." + str(best[np])).definition
							else:	raise(nltk.corpus.reader.wordnet.WordNetError)
					except (nltk.corpus.reader.wordnet.WordNetError):
						if (np[-3:] == "ies"):	lex[np] = wordnet.synset(np[:-3] + "y.n." + str(best[np])).definition
						else:	raise(nltk.corpus.reader.wordnet.WordNetError)
				except (nltk.corpus.reader.wordnet.WordNetError):
					if not self.for_Web:
						(alt[np], lex[np]) = self.define(np, "", best[np], True, self.old_topic, self.new_topic)
					else:	pass
		# for x in range(len(tokens)):
		#	word = tokens[x]
		#	tag = pos[x]
		#	if ((tag in self.noun_tags) and (word not in lex)):
		#		(alt[word], lex[word]) = define.define(word, lex[word], best[word], True, self.old_topic, self.new_topic)
		return (alt, best, lex)

	def printEI(self, ifc, thenc, alt, lex, NPdict):
		"""Prints the evidence and inference, with (newly-/un-)defined words."""
		# output = ""
		# for word in ifc.split():
			# output += word + " "
			# if (word in alt):	output += "(" + lex[word] + ") "
		# print output
		# print tokenize.tokenize(output, -1)
		temp = ifc
		for NP in NPdict.values():
			if ((NP in lex) and (NP in alt)):	temp = re.sub(NP, NP + " (" + lex[NP] + ")", temp)
		self.write("EVIDENCE:\t" + temp, self.debug_file)
		self.write("EVIDENCE:\t" + temp, size = 4)
		temp = thenc
		for NP in NPdict.values():
			if ((NP in lex) and (NP in alt)):	temp = re.sub(NP, NP + " (" + lex[NP] + ")", temp)
		self.write("\nINFERENCE:\t" + temp, self.debug_file)
		self.write("\nINFERENCE:\t" + temp, size = 4)
		# output = ""
		# for word in thenc.split():
			# output += word + " "
			# if (word in alt):	output += "(" + lex[word] + ") "
		# print output
		# print tokenize.tokenize(output, -1)
		self.write("", self.debug_file)

	## Replaced with immigrant module.
	# def strip_punc(self, str):
	#	"""Strips the punctuation (esp. periods, commas, and semicolons) from the rear of the clauses."""
	#	if (str.strip()[-1] in [",", ".", ";"]):	str = str.strip()[:-1]
	#	return str

	def call_paranNP(self, str, lex, alt, best, trace = False):
		"""Does the tokenizing, tagging, and substitution necessary before calling define.paranNP(...)."""
		noparans = re.sub(r" ?\(([^)]*)\)", "", str)
		prelim = self.tokenize(noparans, 0)
		prelim_tagging = self.SeqTagger.tag(prelim.split())
		tokens = list()
		tagging = list()
		for (w, t) in prelim_tagging:
			tokens.append(w)
			tagging.append(t)
		if trace:
			self.write(w, self.debug_file)
			self.write(t, self.debug_file)
		allNPs = self.NPs(tokens, tagging, trace)
		(lex, alt) = self.paranNP(allNPs, str, lex, alt, best)
		return (lex, alt, allNPs, noparans)

	def exit(self):
		"""Exits Revelator, doing a modicum of cleanup along the way."""
		self.write("", "screen")
		if self.for_Web:	print "<br clear=all/><h3 align=center><font face=Times size=4 color=DFBFFF>"
		self.write("Thanks for playing Revelator today, %s!  Have a nice day and please come play again sometime soon!\n" % self.player, "screen", face = self.saluFont, textColor = self.saluColor)
		try:
			os.remove(self.dummyFile)
		except:	pass
		try:
			os.remove(self.debug_file)
		except:	pass
		self.LISTEN = True
		if self.for_Web:
			print "<a href=\"startdemo.cgi?Delete=" + self.player + "\">"
			print "Click here to return the sign-in page.<br/>"
			print "</a></font></h3>"
		self.DONE = True
		sys.exit(0)

	def writeDummy(self, altered, words, received, EII_grammar, str, if_clause, then_clause, pos_if, pos_then, allNPs, valid_if, valid_then, cE, cI, playnum):
		"""Writes a dummy file to keep track of persistent data.  The flag is there, just because."""
		vars = shelve.open(self.dummyFile, 'n')
		vars["flag"] = 'Claudia likes "Rosario+Vampire" and "Shugo Chara!".  I like Claudia.  Thus, I like these anime too.'
		vars["tagfile"] = self.tagfile
		vars["grammar"] = self.grammar
		vars["promptings"] = self.promptings
		vars["ideal"] = self.ideal
		vars["defsfile"] = self.defsfile
		vars["old_topic"] = self.old_topic
		vars["promptText"] = self.promptText
		vars["LISTEN"] = self.LISTEN
		vars["DEFINE"] = self.DEFINE
		vars["CLARIFY"] = self.CLARIFY
		vars["DONE"] = self.DONE
		vars["altered"] = altered
		vars["words"] = words
		vars["seqtagger"] = self.SeqTagger
		vars["train_data"] = self.train_data
		vars["messages"] = self.messages
		vars["transforms"] = self.transforms
		vars["received"] = received
		vars["eii_grammar"] = EII_grammar
		vars["eii_earley"] = self.EII_Earley or None
		vars["str"] = str
		vars["if_clause"] = if_clause
		vars["then_clause"] = then_clause
		vars["pos_if"] = pos_if
		vars["pos_then"] = pos_then
		vars["allnps"] = allNPs
		vars["if"] = valid_if
		vars["then"] = valid_then
		vars["choiceI"] = cI
		vars["choiceE"] = cE
		vars["choicesinference"] = self.choicesinference
		vars["choicesevidence"] = self.choicesevidence
		vars["headinginference"] = self.headinginference
		vars["headingevidence"] = self.headingevidence
		vars["playnum"] = playnum
		vars["player"] = self.player
		vars.close()
	

## IMMIGRATED CODE

## Code immigrated from answer.py.

	def write(self, text, dest = None, face = None, textColor = None, size = "3"):
		"""Outputs the desideratum from the Tutor to STDOUT (or shows what is going to Revelator, for the demo)."""
		if not face:	face = self.font
		if not textColor:	textColor = self.color
		if self.for_Web:
			if (dest == "screen"):
				print "<font face=%s color=%s size=%s>" % (face, textColor, size) + text + "</font><br/>" 
			elif not dest:
				dest = self.output_file
				out = open(dest, 'a')
				out.write("<font face=%s color=%s size=%s>" % (face, textColor, size) + text + "</font><br/>" + "\n")
				out.close()
			else:
				out = open(dest, 'a')
				out.write(text + "\n")
				out.close()
		else:
			print text

## Code immigrated from ask.py.

	def getPlay(self, prompt, type = "text", EI = None):
		"""Gets the play from the user."""
		if self.for_Web:
			## There are a few types of input I wish to allow here.  The default is a text box, used mainly for normal plays.
			if (type == "text"):
				self.write("<b><i>" + prompt + "</i></b>", "screen")
				if ("play" in self.form.keys()):
					received = self.form["play"].value
					return received
			## I can also use radio buttons for mutually-exclusive choices.
			if (type == "radio"):
				self.write("<u><b>Please choose from the following:</b></u><br/>", "screen")
				choices = prompt.split("\n")[:-1]			
				x = 1		## <-- This is so that a return value of 0 isn't treated as a Fallacy
				self.write("<form>", "screen")
				for opt in choices:
					self.write(opt + "<input type=radio name=\"choice" + EI + "\" value=" + str(x) + "/><br/>", "screen")
					x += 1
				self.write("None of the above.", "screen")
				self.write("<input type=radio name=\"choice" + EI + "\" value=-2/><br/>", "screen")
				self.write("<button type=submit name=\"dummyRS\">Resubmit play</button><br/>", "screen")
				self.write("</form>", "screen")
				# self.write(self.form.keys())
				text = "choice" + EI
				if (text in self.form.keys()):
					received = int(self.form[text].value)
					return received
				else:	pass	#return -2
		else:
			return (raw_input(prompt))

## Code immigrated from module bestdefs.py.

	def getBest(self, idealsfile):
		"""Initializes the dictionary of "ideal" definitions for the NLP Tutor."""
		best = dict()
		try:
			file = open(idealsfile, 'r')
			for pair in file:
				best[" ".join(pair.split()[:-1])] = pair.split()[-1]
			file.close()
		except:
			self.write("Dictioary file %s does not exist." % idealsfile, "screen")
		return best

## Code immigrated from module commands.py.
	
	def help(self):
		"""F1"""
		self.write("""
        To make a valid play, simply type a statement in a form resembling the "If..., then..." proposition.<br/>
        A valid play will behave grammatically, and use words that the Tutor knows.  Be prepared to confirm<br/>
that we have the same definition of nouns that you have.<br/>
        If you wish to use a word that I don't know, you will be asked to define it.<br/>
        If you want to skip that step, and define a word on your own, just put its definition in parantheses<br/>
after it.<br/>
        And, if you would prefer to leave it undefined, don't put anything in the parantheses but elipsis<br/>
(...).<br/>
        Finally, if a sentence can be read more than one way, I'll tell you and ask you to rephrase it.<br/>
        All in all, I want you to be able to make plays which will contribute positively to the question<br/>
being begged.  As long as everything is in order, I'm sure you'll be able to make some very valid points and<br/>
have your theses stand the test of time.<br/>
		""", "screen")
	
	def version(self):
		"""Version info.  If you wish to update these, I ask that you leave the old info intact."""
		## Version 1.0.0:  September 10, 2009; Josh "Lain" Johnson
		## Version 1.0.1:  September 14, 2009; Josh "Lain" Johnson
		## Version 1.0.2:  September 30, 2009; Josh "Lain" Johnson
		## Version 1.0.3:  October 13, 2009; Josh "Lain" Johnson
		## Version 1.0.4:  October 19, 2009; Josh "Lain" Johnson
		## Version 1.0.5:  December 11, 2009; Josh "Lain" Johnson
		self.write('NLP Tutor, version 1.0.5.  Created by Josh "Lain" Johnson on 11 December 2009.<br/>', "screen")
	
	def about(self):
		"""Tells a little bit about the Tutor."""
		self.version()
		self.write("""The NLP Tutor helps players to make valid plays, in order to interface effectively
with Revelator.<br/>""", "screen")
	
	def commands(self):
		"""List of commands."""
		self.write("""LIST OF COMMANDS:\n<br/>
        "help"       -- see how to make valid plays in Revelator using this Tutor<br/>
        "version"    -- get version info about this Tutor<br/>
        "about"      -- learn what the Tutor is<br/>
        "sample"     -- see what a sample sentence would look like<br/>
        "quit"       -- leave me by the wayside<br/>
        "commands"   -- what got you here<br/>
		""", "screen")
	
	def sample(self):
		"""Show sample plays (which will not be passed on to Revelator)."""
		self.write("""
        SAMPLE PLAYS<br/>
        ------------<br/>
	<br/>
        Normal play:<br/>
	<br/>
            If the person is over the age of 62, then s/he may retire from the workforce.<br/>
	<br/>
        You may be asked to define words which I don't know.  If you want to define a<br/>
word without being asked, just flank its definition with parantheses, as follows:<br/>
	<br/>
            If the Vikings defeat the Packers in the NFC Championship, then it will upset<br/>
                a lot of cheeseheads (Packers fans).<br/>
		<br/>
        You could also leave a word undefined, for whatever reasons, by putting ellipses<br/>
in the parantheses.<br/>
	<br/>
            If this widget (...) performs the task it should, then our 3rd-quarter revenue<br/>
                will increase threefold.<br/>
	<br/>
        Of course, you are not bound to use "if...then" statements with me.  As long as I<br/>
can rephrase it to an "if...then" statement, then I am satisfied.<br/>
	<br/>
            Since Seras Victoria is bitten by Alacard in the first episode, therefore she<br/>
                becomes his servant for the remainder of the series.<br/>
	<br/>
        I hope this helps you to make satisfactory plays.  Remember that, the better your<br/>
plays conform to my few rules, the faster you will be able to contribute, and the likelier<br/>
your plays are to persist as more plays are made.<br/>
		<br/>
		""", "screen")
	
	def quit(self, string):
		"""Quits the Tutor."""
		return (string.lower() in ["quit", "q", "exit", "stop", "bail", "x"])
	
	def error(self, command):
		"""Prints an error message relative to the command set."""
		self.write("""I'm sorry, but "%s" is not a valid command.  Please type one of my standard commands, make a play, or type "quit" to exit.\n<br/>""" % command, "screen")
		self.commands()
	
	def aCommand(self, command):
		"""Determines whether something is a command.  If so, execute it.  If it is neither that nor a play, throw an error."""
		if (len(command.split()) > 1):	return 0
		else:
			if (not self.quit(command)):
				command = self.convert(command)
				self.write("")
				try:
					exec("self.%s()" % command)
				except(Exception):
					self.error(command)
				return 1
			else:	return -1
	
	def convert(self, command):
		"""Converts a synonymous command with one of the standard command names for the Tutor."""
		if (command.lower() in ["?", "help", "h", "question", "f1"]):	command = "help"
		if (command.lower() in ["about", "a"]):	command = "about"
		if (command.lower() in ["ver", "version", "v", "release"]):	command = "version"
		if (command.lower() in ["sample", "s", "example", "ensample"]):	command = "sample"
		return (command)
	
	def commandsTest1():
		import sys
		commands()
		help()
		version()
		about()
		sample()
		test = "version"
		exec("self.%s()" % test)
		try:
			test = "ver"
			exec("self.%s()" % test)
		except(Exception):
			error(test)
	
	def commandsTest2():
		comm = ""
		while (not self.quit(comm)):
			if (not self.aCommand(comm)):	print "%s was not a command." % comm
			comm = self.getPlay("Your next command:  ")
	
## Code immigrated from module define.py.
	
	def yes(self, string):
		"""Returns True if the string is an affirmation."""
		if (string.lower() in ["y", "yes", "yeah", "yup", "ok"]):	return True
		else:	return False
	
	def define(self, word, Webster, bestdef, changed, old_topic, new_topic):
		"""Defines a word, if desired by the user, and if the topic has changed."""
		if ((Webster != "") and (not changed)):	return (False, Webster)
		if (Webster == ""):
			self.write("The word " + word + " was not defined under the topic " + old_topic + ".")
		else:
			asked = self.getPlay("The word " + word + " was defined under the topic " + old_topic + " as " + Webster + ".\nDo you want this meaning to carry over to the new topic " + new_topic + "?  ")
			if self.yes(asked):
				return (False, Webster)
		undone = True
		dno = 1
		while (undone):
			if (dno == bestdef):	dno += 1
			string = word + ".n." + str(dno)
			try:
				if (dno < len(wordnet.synsets(word, pos = wordnet.NOUN))):
					asked = self.getPlay("Does " + wordnet.synset(string).definition + " work for your usage of " + word + "?  ")
					undone = not yes(ask)
					newdef = wordnet.synset(string).definition
					dno += 1
				else:
					newdef = self.getPlay("Then how would you define " + word + "?  ")
					undone = False
			except(Exception):
				newdef = self.getPlay("How would you define " + word + "?  ")
				undone = False
		return (True, newdef)
	
	def paran(self, str, lex, alt, best):
		"""Allows the user to self-define a word by putting its definition in parentheses."""
		allfinds = re.finditer(r"(\w+) ?\(([^)]*)\)", str)
		for found in allfinds:
			word = found.group(1)
			meaning = found.group(2)
			# print '"' + word + '" is defined here as "' + meaning + '".'
			if ((meaning == "...") or (meaning == "")):
				if (word in best):
					lex[word] = wordnet.synset(word + ".n." + repr(best[word])).definition
				else:
					lex[word] = "HERETOFORE UNDEFINED"
			else:
				lex[word] = meaning
			alt[word] = True
			str = re.sub(re.compile(" ?\(" + meaning + "\)"), "", str)
		return (str, lex, alt)
	
	def paranNP(self, NPs, str, lex, alt, best):
		"""Allows the user to self-define a mutable noun phrase by putting its definition in parentheses."""
		for nouns in NPs.values():
			found = re.search(re.compile("(" + nouns + ") ?\(([^)]*)\)"), str)
			if (not found):	continue
			# print "Found", nouns
			word = found.group(1)
			meaning = found.group(2)
			# print '"' + word + '" is defined here as "' + meaning + '".'
			if ((meaning == "...") or (meaning == "")):
				if ((word in best) and (word + ".n." + repr(best[word]) in wordnet.synset)):
					lex[word] = wordnet.synset(word + ".n." + repr(best[word])).definition
				else:
					lex[word] = "HERETOFORE UNDEFINED"
			else:
				lex[word] = meaning
			alt[word] = True
		return (lex, alt)
	
	def paranest(self, str, lex, alt):
		"""Allows the user to self-define a word by putting its definition in parentheses.
			This is the recursive counterpart to parans(...) above."""
		allfinds = re.finditer(r"(\w+) \(([^)]*)\)", str)
		for found in allfinds:
			word = found.group(1)
			meaning = found.group(2)
			# print '"' + word + '" is defined here as "' + meaning + '".'
			if (re.search(r"\(", meaning)):	(meaning, lex, alt) = paranest(meaning, lex, alt)
			lex[word] = meaning
			alt[word] = True
			str = re.sub(re.compile(" \(" + meaning + "\)"), "", str)
			# print str
		return (str, lex, alt)
	
	def RB(self, tokens, tags, lex, alt, best):
		"""Just like paran(), this lets the user self-define a word; only, this happens after tokenization and tagging."""
		x = 0
		while (x < len(tags)):
			meaning = ""
			if (tags[x] in ["-LRB-", "("]):
				x += 1
				while (tags[x] not in ["-RRB-", ")"]):
					meaning += tags[x] + " "
					if (x > len(tags)):	break
					x += 1
			else:	x += 1
			if (meaning == "... "):
				word = tokens[x - 1]
				if (word in best):
					lex[word] = wordnet.synset(word + ".n." + repr(best[word])).definition
				else:
					lex[word] = "HERETOFORE UNDEFINED"
				alt[word] = True
			else:
				if (meaning):
					word = tokens[x - 1]
					lex[word] = meaning[:-1]
					alt[word] = True
		return (lex, alt)
	
	def defineTest1(self, words, altered, ideal, old_topic, new_topic, changed):
		"""First test."""
		sent = [["The", "DT"], ["ice", "NN"], ["dilluted", "VBD"], ["the", "DT"], ["cocktail", "NN"], [".", "DOT"]]
		## Initial run:
		for (word, tag) in sent:
			if (tag in ["NN", "NNS", "FW"]):
				# (altered[word], words[word]) = define(word, "")
				altered[word] = True;
				## The ideal definition of a word, optional for now, given a particular topic.
				ideal[word] = 1
				string = word + ".n." + str(ideal[word])
				words[word] = wordnet.synset(string).definition
		for word in sent:
			print word[0], 
			if (word[0] in altered):	print "(" + words[word[0]] + ")",
		print "\n"
		## Unchanged run:
		changed = (new_topic != old_topic)
		if (not changed):	altered = dict()
		if not self.for_Web:
			for (word, tag) in sent:
				if (tag in ["NN", "NNS", "FW"]):
					(altered[word], words[word]) = self.define(word, words[word], ideal[word], changed, old_topic, new_topic)
		for word in sent:
			print word[0], 
			if (word[0] in altered):	print "(" + words[word[0]] + ")",
		print "\n"
		## New topic:
		new_topic = "potpourri"
		changed = (new_topic != old_topic)
		if (not changed):	altered = dict()
		if not self.for_Web:
			for (word, tag) in sent:
				if (tag in ["NN", "NNS", "FW"]):
					(altered[word], words[word]) = self.define(word, words[word], ideal[word], changed, old_topic, new_topic)
		for word in sent:
			print word[0], 
			if (word[0] in altered):	print "(" + words[word[0]] + ")",
		print "\n"
	
	def defineTest2(self, words, altered, ideal):
		"""Second test."""
		test = "I thought that anime (cartoons from Japan) would make a great gift for my GF (girlfriend) ."
		(test, words, altered) = self.paran(test, words, altered, ideal)
		for token in test.split():
			print token,
			if (token in altered):	print "(" + words[token] + ")",
		print
		test = 'I thought that "Rosario+Vampire" (an anime (cartoons from Japan)) would make a great gift for my GF (girlfriend) .'
		# (test, words, altered) = paranest(test, words, altered)
		(test, words, altered) = self.paran(test, words, altered, ideal)
		for token in test.split():
			print token,
			if (token in altered):	print "(" + words[token] + ")",
	
	def defineTest3(self, words, altered, ideal):
		"""Third test."""
		ideal["day"] = 1
		test = "She turned twenty (20) the day (...) before the equinox () on September twenty ."
		(test, words, altered) = self.paran(test, words, altered, ideal)
		for token in test.split():
			print token,
			if (token in altered):	print "(" + words[token] + ")",
		print
	
	def defineTest4(self, lex, alt, best):
		"""4th test."""
		lexes = ["Meet", "the", "jade", "macaque", "(", "monkey", ")", "beneath", "the", "willow", "trees", "of", "the", "Gobe", "Desert", "at", "noon", "-LRB-", "...", "-RRB-", "."]
		poss = ["VB", "DT", "NN", "NN", "-LRB", "NN", "-RRB", "IN", "DT", "NNS", "NNS", "IN", "DT", "FW", "NNP", "IN", "NN", "-LRB-", "SYM", "-RRB-", "DOT"]
		(lex, alt) = self.RB(lexes, poss, lex, alt, best)
		# for 
	
	def defineTest5(self, words, altered, ideal):
		"""750mL test."""
		test = "Beneath the oceans (bodies of water) of Terra (Earth) are mold (...) up to 5 km (kilometers) in thickness()."
		noRBs = re.sub(r" ?\(([^)]*)\)", "", test)
		broken = self.tokenize(noRBs, 0)
		# print broken
		(SeqTagger, td) = self.getTagger('sentences.dat', 0, 4)
		tagged = SeqTagger.tag(broken.split())
		tok = list()
		tag = list()
		for (word, pos) in tagged:
			tok.append(word)
			tag.append(pos)
		# print tok
		# print tag
		NPdict = self.NPs(tok, tag)
		(words, altered) = self.paranNP(NPdict, test, words, altered, ideal)
		for NP in NPdict.values():
			# print "Noun phrase:  " + NP
			noRBs = re.sub(NP, NP + " (" + words[NP] + ")", noRBs)
		print noRBs, "\n"
	
## Code immigrated from module extract.py.
	
	def getEI(self, sentence):
		"""Gets the clausal/phrasal parts of a sentence."""
		ifs = "(if|while|when|given)"
		thens = "(then|therefore|ergo)"
		pattern = ifs + " (.*) " + thens + " (.*)"
		splitup = re.match(pattern, sentence, re.I)
		# print splitup.groups(0)
		return (self.strip_punc(splitup.group(2)), self.strip_punc(splitup.group(4)))
	
	def strip_punc(self, str):
		"""Strips the unncessary punctuation from the rear end of the clauses."""
		## We'll leave this is for more conventional-looking strings.
		if (str.strip()[-1] in [".", ",", ";"]):	str = str.strip()[-1]
		## This takes our type-0 tokenization into account.
		if (str.split()[-1] in ["&comma", "&period", ";", "&colon", "&amp"]):
			str = " ".join(str.split()[:-1])
		return str
	
## Code immigrated from module format.py.
	
	def iscons(self, letter):
		"""Tests whether a letter is a consonant."""
		return (letter.isalpha() and (re.match(r"a|e|i|o|u", letter) == None))
	
	def upper(self, word):
		## ~+##+~
		"""Puts a word in uppercase."""
		return word.capitalize()
	
	def lower(self, word):
		## ~_##_~
		"""Puts a word in lowercase."""
		return word.lower()
	
	def poss(self, word):
		## ~$##$~
		"""Converts a PRP into its PRP$."""
		word = word.lower()
		if (word == "it"):	return "its"
		if (word == "you"):	return "your"
		if (word == "y'all"):	return "y'all's"
		if ((word == "him") or (word == "he")):	return "his"
		if ((word == "her") or (word == "she")):	return "her"
		if ((word == "i") or (word == "me")):	return "my"
		if ((word == "we") or (word == "us")):	return "our"
		if ((word == "they") or (word == "them")):	return "their"
	
	def gerund(self, word):
		## ~&##&~
		"""For converting a word from VB to VBG."""
		word = word.lower()
	
		## Rule 0:  Irregulars.
		if (word == "be"):	return "being"
	
		## Rule 1:  String the silent "e" first.
		if ((word[-1] == "e") and (self.iscons(word[-2]))):
			return (word[:-1] + "ing")
	
		## Rule 2:  Double the final consonant first.
		if (not self.iscons(word[-2]) and self.iscons(word[-1]) and (word[-3] != word[-2])):
			return (word + word[-1] + "ing")
	
		## Otherwise, just append "ing".
		return (word + "ing")
	
	def base(self, word):
		## ~@##@~
		"""Convert a VBZ into a VB."""
		word = word.lower()
	
		## First, we resolve irregular verbs.
		if (word == "is"):	return "be"
		if (word == "has"):	return "have"
		
		## Next, words ending in "ies".
		if (word[-3:] == "ies"):	return (word[:-3] + "y")
	
		## Then, words ending in "es".
		if ((word[-2:] == "es") and (not self.iscons(word[-3]) or self.iscons(word[-4]))):
			return word[:-2]
	
		## All other verbs now.
		return (word[:-1])
	
	def base2(self, word):
		## ~@##@~
		"""Convert a VBG into a VB."""
		word = word.lower()[:-3]
	
		## First, words with double consonants.
		if (self.iscons(word[-1]) and (word[-2] == word[-1])):	return word[:-1]
		
		## Next, words with a silent "e".
		if (self.iscons(word[-1]) and not self.iscons(word[-2]) and ((len(word) < 3) or (word[-3] != word[-2]))):
			return (word + "e")
	
		## All other words now.
		return word
	
	def passive(self, word):
		## ~|##|~
		"""A word is being converted here into its passive form from its VBZ form."""
	
		word = word.lower()
	
		## We will ignore the vast majority of irregular verbs at present, using only what we need.
		## Exceptions:
		if (word == "is"):	return "being"
		if (word == "becomes"):	return word
		if (word == "begins"):	return "begun"
		if (word == "bends"):	return "bent"
		if (word == "breaks"):	return "broken"
		if (word == "chooses"):	return "chosen"
		if (word == "does"):	return "done"
		if (word == "falls"):	return "fallen"
		if (word == "gets"):	return "gotten"
		if (word == "goes"):	return "gone"
		if (word == "gives"):	return "given"
		if (word == "was"):	return ""
		if (word == "throws"):	return "thrown"
		if (word == "catches"):	return "caught"
	
		## Verbs ending in "es":
		if (word[-2:] == "es"):	return (word[:-1] + "d")
	
		## Default:
		return (self.gerund(self.base(word))[:-3] + "ed")
	
	def bool(self, word, adverb):
		"""Performs a minor Boolean change."""
		word = word.lower()
		adverb = adverb.lower()
		if (adverb == "not"):
			if (word == "or"):	return ("and", adverb)
			if (word == "and"):	return ("or", adverb)
		return (word, adverb)
	
## Code immigrated from module getnps.py.
	
	def NPs(self, words, tags, trace = False):
		"""Gets the NP's from a tagged sentence."""
		if (len(words) != len(tags)):
			print "Tagging error.  There are different numbers of tokens than there are tags."
			return list()
		nps = dict()
		x = 0
		if trace:	
			print words, tags, len(words), len(tags)
		while (x < len(tags)):
			coll = ""
			if (tags[x] in ["NN", "NNS", "NNP", "NNPS", "FW"]):
				while (tags[x] in ["NN", "NNS", "NNP", "NNPS", "FW"]):
					if trace:	print words[x], tags[x]
					if trace:	print x
					coll += words[x] + " "
					x += 1
					if (x == len(tags)):	break
			else:	x += 1
			if (coll):	nps[x - 1] = coll[:-1]
		return nps

## Code immigrated from module prompt.py.
	
	def readout(self, leaves):
		"""Gives a readout of the leaves of a (sub)tree as if it were a sentence fragment."""
		out = ""
		for leaf in leaves:	out += leaf + " "
		return out.rstrip()
	
	def init_messages(self, file):
		"""Prepares the list of messages for prompting the user when s/he has an ambiguity in the input, and the list of 
			templates for the sentences once the user chooses which format s/he prefers."""
		herald = open(file, 'r')
		message = dict()
		converts = dict()
		for news in herald:
			if ((news[0] != "#") and (news[0] != "\n")):
				# print news
				if (news.split("::")[0].strip() in message):
					message[news.split("::")[0].strip()].append(news.split("::")[1].strip())
				else:
					message[news.split("::")[0].strip()] = list([news.split("::")[1].strip()])
				if (news.split("::")[0].strip() in converts):
					converts[news.split("::")[0].strip()].append(news.split("::")[2].strip())
				else:
					converts[news.split("::")[0].strip()] = list([news.split("::")[2].strip()])
		# message["DT FW VP "] = 'Do you mean that "~~~0~~~ ~~~1~~~" or that "~~~0~~~ has ~~~1~~~"?'
		return (message, converts)
	
	def init_sents(self, file):
		"""Prepares the list of sample sentences for testing the prompts."""
		herald = open(file, 'r')
		message = dict()
		for news in herald:
			if ((news[0] != "\n") and (news[0] != "#")):
				message[news.split("::")[0].strip()] = news.split("::")[1].strip()
		return message
	
	def toTuple(self, string):
		"""Converts a string in the form "#,#,#,#,...,#" into an ordered tuple."""
		vector = tuple()
		for num in string.split(","):
			vector += tuple([int(num)])
		return vector
	
	def fitInto(self, tags, message):
		"""Fits the tagged terminals into the message template."""
		forms = "((\-|\||&|_|\+|\^|\$|@|=))"
		pattern = "~" + forms + "?#" + forms + "?~"
		madlib = message
		broken = tags.split()
		for x in range(len(broken)):
			label = re.sub("#", str(x), pattern)
			insert = broken[x]
			found = re.search(label, madlib)
			if found:	
				flag = str(found.group(1))[0]
				if (flag == "N"):	pass			## Corresponds to "None"
				if (flag == "_"):	insert = self.lower(insert)
				if (flag == "+"):	insert = self.upper(insert)
				if (flag == "-"):	insert = self.base(insert)
				if (flag == "="):	insert = self.base2(insert)
				if (flag == "@"):	insert = self.poss(format.lower(insert))
				if (flag == "^"):	insert = self.bool(insert, broken[x - 2])
				if (flag == "$"):	insert = self.poss(insert)
				if (flag == "|"):	insert = self.passive(insert)
				if (flag == "&"):	insert = selff.gerund(insert)
				madlib = re.sub(label, insert, madlib)
		return madlib
	
	def reformat(self, sentence, template):
		"""Reformats a sentence, converting it to fit in the given format."""
		words = sentence.split()
		new = ""
		for spot in template.split():
			 new += words[sport] + " "
		return new[:-1]
	
## Code immigrated from module seqtagger.py.
	
	def getTagger(self, file, NUM_SENTS = 0, N = 4):
		"""Creates, trains, and returns a Sequential tagger."""
		default_tagger = nltk.DefaultTagger("NN")
		tagfile = open(file, 'r')
		sent = list()
		token_seq = list()
		tag_seq = list()
		training_data = list()
		RCE = 0
		for entry in tagfile:
			if ((NUM_SENTS != 0) and (NUM_SENTS == RCE)):	break
			if (entry == "\n"):
				training_data.append(sent)
				sent = list()
				continue
			entry = self.tokenize(entry, 3).strip()
			sent.append(tuple([" ".join(entry.split()[:-1]), entry.split()[-1]]))
			token_seq.append(" ".join(entry.split()[:-1]))
			tag_seq.append(entry.split()[-1])
			RCE += 1
		uni = nltk.UnigramTagger(train=training_data, backoff=default_tagger)
		bi = nltk.BigramTagger(train=training_data, backoff=uni)
		tri = nltk.TrigramTagger(train=training_data, backoff=bi)
		return (nltk.NgramTagger(N, train=training_data, backoff=tri), training_data)
	
## Code immigrated from module tokenize.py.
	
	def tokenize(self, s, type = 0):
		"""
			Tokenize a string.
	
			type == -1: Undoes standard text tokenization
			type == 0:  Standard text tokenization
			type == 1:  Tokenization for the tree grammar
			type == 2:  Tokenization for weighing the files for classification
			type == 3:  Tokenization for the CRF file
			type == 4:  Grammar (.cfg) file
		"""
		not_incl = "([^\w\d\s\.\-/,':;&\(\)])"
		if (type == -1):
			s = re.sub(" &ellie ", "...", s)
			s = re.sub(" - ", "-", s)
			s = re.sub(" -- ", " - ", s)
			s = re.sub(" -LRB- ", "(", s)
			s = re.sub(" -RRB- ", ")", s)
			s = re.sub(r"(\w+) ('\w\w?)", r"\1\2", s)
			s = re.sub(" / ", "/", s)
			s = re.sub(" &period ", ".", s)
			s = re.sub(" &comma", ",", s)
			s = re.sub(" &dollar ", " $", s)
			s = re.sub(" &colon ", ": ", s)
			s = re.sub(" &amp ", " & ", s)
			s = re.sub(" ; ", ";", s)
			s = re.sub(re.compile(not_incl + " "), r'\1', s)
		if (type == -99):	## It's faster to ignore it this way.
			s = re.sub(r"\.", " &period ", s)
			s = re.sub(r"\-+", "-", s)
			s = re.sub("-", " - ", s)
			s = re.sub(r"(\w+)'s", r"\1 's", s)
			s = re.sub("/", " / ", s)
			s = re.sub(",", " &comma ", s)
			s = re.sub(":", " &colon ", s)
			s = re.sub(r"\(", " -LRB- ", s)
			s = re.sub(r"\)", " -RRB- ", s)
		if (type == 1):
			s = re.sub(r"\.\)", "&period)", s)
			s = re.sub(r",\)", "&comma)", s)
			s = re.sub(r"\(\.", "(DOT", s)
			s = re.sub(r"\(,", "(COMMA", s)
			s = re.sub(r"\$,", "(DOLLAR", s)
			s = re.sub(r"\$\)", "&dollar)", s)
			s = re.sub(r":\)", "&colon)", s)
			s = re.sub(r"\(:", "(COLON", s)
		if ((type == 2) or (type == 0)):
			s = re.sub(r"&", " &amp ", s)
			s = re.sub(r"\.{3}", " &ellie ", s)
			s = re.sub(r"\.", " &period ", s)
			s = re.sub(r"\-+", "-", s)
			s = re.sub(" - ", " -- ", s)
			s = re.sub(r"(\S)-(\S)", r"\1 - \2", s)
			s = re.sub(r"(\w+)('\w?\w?)", r"\1 \2", s)
			s = re.sub("/", " / ", s)
			s = re.sub(", ", " &comma ", s)
			s = re.sub(r",(\S)", r" &comma \1", s)
			s = re.sub(r"\$", " &dollar ", s)
			s = re.sub(":", " &colon ", s)
			s = re.sub(";", " ; ", s)
			s = re.sub(r"\(", " -LRB- ", s)
			s = re.sub(r"\)", " -RRB- ", s)
			## Pad other punctuation with a subsequent spaces:
			s = re.sub(re.compile(not_incl), r'\1 ', s)
		if (type == 3):
			if ((s == "") or (re.match("%", s))):	return s
			s = re.sub(r"\(", " -LRB- ", s)
			s = re.sub(r"\)", " -RRB- ", s)
			s = re.sub(r"^\.{3} .*", "&ellie CRAFT", s)
			tt = s.split()
			## For tokens only:
			tt[0] = re.sub(r"^\s&\s", " &amp ", tt[0])
			tt[0] = re.sub(r"^\.", "&period", tt[0])
			tt[0] = re.sub("^,", "&comma", tt[0])
			tt[0] = re.sub("^:", "&colon", tt[0])
			tt[0] = re.sub(r"^\$", "&dollar", tt[0])
			## For tags only:
			tt[1] = re.sub(r"PRP\$$", "PRPS", tt[1])
			tt[1] = re.sub(r"WP\$$", "WPS", tt[1])
			tt[1] = re.sub(r"\.$", "DOT", tt[1])
			tt[1] = re.sub(",$", "COMMA", tt[1])
			tt[1] = re.sub(":$", "COLON", tt[1])
			tt[1] = re.sub(r" \$$", " DOLLAR ", tt[1])
			s = string.join(tt, " ")
		if (type == 4):
			if ((s == "") or (re.match("%", s))):	return s
			s = re.sub(r"\(", "-LRB-", s)
			s = re.sub(r"\)", "-RRB-", s)
			s = re.sub(r"& ", "&amp ", s)
			s = re.sub(r"PRP\$", "PRPS", s)
			s = re.sub(r"WP\$", "WPS", s)
			s = re.sub(r"\"\$\"", "\"&dollar\"", s)
			s = re.sub(r"\"\.\"", "\"&period\"", s)
			s = re.sub("\",\"", "&comma", s)
			s = re.sub("\":\"", "&colon", s)
			s = re.sub(r"\.", "DOT", s)
			s = re.sub(r"\$", "DOLLAR", s)
			s = re.sub(",", "COMMA", s)
			s = re.sub(":", "COLON", s)
		return (s)

# if (__name__ == "__main__"):
# if (len(sys.argv) > 7):	Allen1 = sys.argv[7]
# if (len(sys.argv) > 6):	purpose = sys.argv[6]
# if (len(sys.argv) > 5):	topic = sys.argv[5]
# if (len(sys.argv) > 4):	best = sys.argv[4]
# if (len(sys.argv) > 3):	prmz = sys.argv[3]
# if (len(sys.argv) > 2):	cfg = sys.argv[2]
# if (len(sys.argv) > 1):	crf = sys.argv[1]
## Uncomment these for the Python file.
# print crf, cfg, prmz, best, topic, purpose, Allen1
# Tutor(crf, cfg, prmz, best, topic, int(purpose), int(Allen1))
Tutor("demo.dat", "grammar.cfg", "allambis2.prmz", "gemology.best", "Gemology", 2, 0)
